﻿import multiprocessing
import multiprocessing.managers
import multiprocessing.synchronize
import time
import traceback
import math
import typing
import configparser
import requestNovelsLib
import multiprocessing
import datetime
import lxml
import lxml.etree
import requests
import sys

## %Y-%m-%d %H:%M:%S
timeFmt = "%m-%d"

__runMain = False

__maxNovelCount = -1
_pageinfocount = 20000
__var: multiprocessing.managers.ValueProxy[int] = None
__typeCount: multiprocessing.managers.ValueProxy[int] = None
__novelMapCount: multiprocessing.managers.ValueProxy[dict[str, int]] = None


def getNovelHome(response: requests.Response) -> typing.Union[dict[str, list[str]], None]:
	"""
	首页查找到类型与 url 的映射
	:param response: get请求之后而的响应体
	:return: 类型名称与之匹配的 url 映射
	"""
	global __var, __typeCount
	text = requestNovelsLib.getAutoCodeText(response)
	xpath = requestNovelsLib.XpathHtml(text)
	## 获取 li 节点
	liNodes = xpath.xpath(r'//ul/li/a')
	if xpath.resultLen() == 0:
		requestNovelsLib.writeLogFile("没有找到任何类型", response.url)
		return
	## 类型名过滤器
	getTypeName = requestNovelsLib.getNovelTypeName()
	
	## 返回
	result: dict[str, list[str]] = {}
	## 遍历需要的类型名称
	typeNameUrl = ""
	href: str = ""
	startIndex = -1
	for liNode in liNodes:
		liNodeXpathInfo = requestNovelsLib.XpathInfo(liNode)
		aTests = liNodeXpathInfo.xpath(r'./text()')
		if liNodeXpathInfo.resultLen() == 0:
			continue
		typeName = aTests[0]
		typeName = requestNovelsLib.strRemoveSpaces(typeName)
		for item in getTypeName:
			if item == typeName:
				href = liNodeXpathInfo.xpath(r'./@href')[0]
				startIndex = href.find("/")
				if startIndex != -1:
					href = href[startIndex + 1:]
				typeNameUrl = requestNovelsLib.getResponseAtStr(response, href)
				break
		if len(typeNameUrl) != 0:
			if result.get(typeName) is None:
				result[typeName] = [typeNameUrl]
			else:
				result[typeName].append(typeNameUrl)
			typeNameUrl = ""
	__var.value = requestNovelsLib.getWorkerThreadNumber()
	__typeCount.value = len(result)
	return result


def getPageListNovel(response: requests.Response, getCount: int, novelTypeName, novelTypeUrl, novelRootUrl) -> typing.Union[list[requestNovelsLib.NovelInfo], None]:
	"""
	获取小说信息
	若为 none 或 数量 0， 则退出这次类型的扫描
	:param response: get 之后的响应体
	:param getCount: 已经库存的数量
	:param novelTypeName: 小说类型名称
	:param novelTypeUrl: 小说类型 url
	:param novelRootUrl: 小说网站
	:return: 小说列表
	"""
	global __maxNovelCount, __runMain
	xpath = requestNovelsLib.XpathHtml(response.text)
	liNodes = xpath.xpath(r'//div[@class="l"]/ul/li')
	if xpath.resultLen() == 0:
		return None
	## 小说网址
	novelUrl: str = ""
	## 小说名称
	novelName: str = ""
	## 小说作者
	author: str = ""
	## 最后更新日期
	updateTime: str = ""
	## 最后更新选项
	lastItem: str = ""
	## 小说详情
	info: str = ""
	## 小说页面请求
	novelInfoPageRequestResult = None
	## 小说页面请求xpath
	novelInfoPageXpathResult = None
	## 返回值
	result: list[requestNovelsLib.NovelInfo] = []
	novelItemCount = 0
	mod = 0
	removeTimeStrLen = len("最后更新：")
	for liNode in liNodes:
		novelUrl = novelName = novelName = author = updateTime = lastItem = info = ""
		novelInfoPageRequestResult = None
		novelInfoPageXpathResult = None
		
		try:
			nodeXpathInfo = requestNovelsLib.XpathInfo(liNode)
			
			## 获取名称
			nodeTextArray = nodeXpathInfo.xpath(r'./span[@class="s2"]/a/text()')
			if nodeXpathInfo.resultLen() == 0:
				print(f"错误的 => 名称 [{novelTypeName} ({getCount})] error : error ==== >")
				continue
			novelName = nodeTextArray[0]
			novelName = novelName.strip()
			
			## 获取链接
			nodeTextArray = nodeXpathInfo.xpath(r'./span[@class="s2"]/a/@href')
			if nodeXpathInfo.resultLen() == 0:
				print(f"错误的 => 链接 [{novelTypeName} ({getCount})] {novelName} :  error ==== >")
				continue
			novelUrl = nodeTextArray[0]
			
			## 获取时间
			nodeTextArray = nodeXpathInfo.xpath(r'./span[@class="s3"]/text()')
			resultLen = nodeXpathInfo.resultLen()
			if resultLen == 0:
				print(f"错误的 => 日期 [{novelTypeName} ({getCount})] error : error ==== >")
				continue
			updateTime = nodeTextArray[0]
			updateTime = updateTime.strip()[1:-1]
			if not requestNovelsLib.checkUpDate(updateTime, timeFmt):  ## 时间跳出
				print(f"=> 链接 [{novelTypeName} ({getCount})] {novelName} :  error ==== > 过期")
				continue
			
			## 获取最后更新项
			nodeTextArray = nodeXpathInfo.xpath(r'./span[@class="s3"]/a/text()')
			if nodeXpathInfo.resultLen() == 0:
				continue
			lastItem = nodeTextArray[0]
			lastItem = lastItem.strip()
			
			## 获取作者
			nodeTextArray = nodeXpathInfo.xpath(r'./span[@class="s5"]/text()')
			if nodeXpathInfo.resultLen() != 0:
				author = nodeTextArray[0]
				author = author.strip()
			else:
				print(f"错误的 => 作者 [{novelTypeName} ({getCount})] {novelName} : {novelUrl} ==== >")
			
			print(f"发现 [{novelTypeName} ({getCount})] {novelName} : {novelUrl} ==== >")
			## 添加单个对象
			result.append(requestNovelsLib.NovelInfo(novelRootUrl=novelRootUrl, novelTypeName=novelTypeName, novelTypeUrl=novelTypeUrl, novelUrl=novelUrl, novelName=novelName, author=author, updateTime=updateTime, lastItem=lastItem, info=info, updateTimeFmt=timeFmt))
			getCount += 1
		
		except:
			requestNovelsLib.writeLogFile(traceback.format_exc(), response.url, response)
			traceback.print_exc()
			print(f"================>\n\t{novelUrl}\n<===========")
	## 返回
	if len(result) == 0:
		return None
	return result


__nextRequeInfoSec = 0.8


def appRequestNovelPage(novelInfo: requestNovelsLib.NovelInfo, response: requests.Response, requestGetUrlDate: datetime, getCount=0) -> requestNovelsLib.NovelInfo:
	"""
	用于请求小说页面
	:param novelInfo: 小说内容
	:param response: 小说详细页面
	:param requestGetUrlDate: 小说请求时候的时间
	:param getCount: 当前有效个数
	:return: 完整小说
	"""
	global __nextRequeInfoSec, __var
	if response is None:
		return None
	## 获取小说内容
	xpath = requestNovelsLib.XpathHtml(response.text)
	liNodes = xpath.xpath(r'//div[@id="info"]')
	if xpath.resultLen() == 0:
		return None
	try:
		## 子节点
		nodeXpathInfo = requestNovelsLib.XpathInfo(liNodes[0])
		
		## 更新日期
		nodeResult = nodeXpathInfo.xpath(r'./p/text()')
		if nodeXpathInfo.resultLen() == 0:
			## print(f"[{novelInfo.novelTypeName}]{novelInfo.novelName} : {novelInfo.novelUrl} ==> 日期错误 errir code : -1")
			return None
		nodeBuffStr: str = nodeResult[-3]
		nodeBuffStr = nodeBuffStr.replace("最后更新：", "").strip()
		novelInfo.updateTime = nodeBuffStr
		if not requestNovelsLib.checkUpDate(nodeBuffStr, timeFmt):
			## print(f"[{novelInfo.novelTypeName}]{novelInfo.novelName} : {novelInfo.novelUrl} ==> 过期  errir code : -2")
			return None
		## 作者
		nodeBuffStr = nodeResult[0]
		findIndex = nodeBuffStr.find("：")
		if findIndex != 0:
			nodeBuffStr = nodeBuffStr[findIndex + 1:]
		novelInfo.author = nodeBuffStr
		## 最后更新项
		nodeResult = nodeXpathInfo.xpath(r'./p/a/text()')
		if nodeXpathInfo.resultLen() == 0:
			## print(f"[{novelInfo.novelTypeName}]{novelInfo.novelName} : {novelInfo.novelUrl} ==> 最后更新项错误  errir code : -3")
			return None
		novelInfo.lastItem = nodeResult[2]
		## 详细信息
		liNodes = xpath.xpath(r'//div[@id="intro"]/p/text()')
		if xpath.resultLen() == 0:
			## print(f"[{novelInfo.novelTypeName}]{novelInfo.novelName} : {novelInfo.novelUrl} ==> 详细信息错误  errir code : -4")
			return None
		nodeBuffStr = liNodes[0]
		novelInfo.info = nodeBuffStr.strip()
		print(f"存档 ({novelInfo.novelName}) [{novelInfo.updateTime}] : {novelInfo.novelUrl} [{novelInfo.novelTypeName}] : {getCount + 1} / {__maxNovelCount}")
		return novelInfo
	except:
		requestNovelsLib.writeLogFile("小说获取异常", novelInfo.novelRootUrl)
		sys.stderr.write(traceback.format_exc())
	finally:  ## 比较时间
		currentDate = nodeBuffStr = datetime.datetime.now()
		modDate = currentDate - requestGetUrlDate
		sec: int = int(modDate.total_seconds() + 0.91)
		sleepSec = __nextRequeInfoSec * __var.value
		if sec < sleepSec:
			sec = sleepSec - sec
			time.sleep(sec)
	return None


typeCountMap = {}
typePageUrlMap = {}


def getPageNextUrl(response: requests.Response, exsiNovel: int, noveList, appenLen, novelTypeName: str, novelTypeUrl: str, novelRootUrl: str) -> typing.Union[str, None]:
	"""
	获取小说翻页的下一页
	:param response: get 之后的响应体
	:param exsiNovel: 已经库存的小说
	:param noveList: 查找到的小说
	:param appenLen: 新增数量
	:param novelTypeName: 小说类型名称
	:param novelTypeUrl: 小说类型 url
	:param novelRootUrl: 小说网站
	:return: 下一页，不存在则返回 none
	"""
	global __maxNovelCount, __runMain, __var, typeCountMap
	if (__maxNovelCount > 0 and __maxNovelCount < exsiNovel) or appenLen == 0:
		return None
	xpath = requestNovelsLib.XpathHtml(response.text)
	nextPage = xpath.xpath(r'//div[@id="pagelink"]/a[@class="next"]/@href')
	if xpath.resultLen() == 0:
		return None
	return nextPage[0]


def sort(novel: requestNovelsLib.NovelInfo):
	return novel.updateTime


__runTime = None


def setRunTime(time):
	global __runTime
	__runTime = time


def getRunTime():
	global __runTime
	return __runTime


def getRootUrl():
	return 'www.biqusk.com'


def endCall(novelTypeName: str, novelTypeUrl: str, novelRootUrl: str, requestUrl: str, isAppendNovel: bool):
	"""
	类型调用结束时候被自动调用
	:param novelTypeName: 类型名称
	:param novelTypeUrl: 类型 url
	:param novelRootUrl: 网站 root
	:param requestUrl: 最后请求 url
	:param isAppendNovel: 追加状态识别码
	"""
	global __var, __typeCount
	__typeCount.value -= 1
	if __typeCount.value < __var.value:
		__var.value = __typeCount.value


def modeRequestGetUrl(lock: multiprocessing.synchronize.Lock = None, workerThreadNumber=0, nextRequestTimeSece=0.05, titleSleepTime=10):
	global __maxNovelCount, __runMain, __var, __typeCount, __novelMapCount, _pageinfocount
	if not __runMain:
		if lock is None:
			raise RuntimeError("请在调用时候设置它的进程锁")
	else:
		if lock is None:
			lock = multiprocessing.Lock()
	
	__var = multiprocessing.Value('i', 0)
	__typeCount = multiprocessing.Value('i', 0)
	manager = multiprocessing.Manager()
	__novelMapCount = manager.dict()
	try:
		__maxNovelCount = int(requestNovelsLib.readIni(requestNovelsLib.getDownLoadIniFilePath(), f"{getRootUrl()}", "count", __maxNovelCount))
	except:
		requestNovelsLib.writeLogFile("无法获取匹配的数量", httpUrl=getRootUrl())
		traceback.print_exc()
	try:
		_pageinfocount = int(requestNovelsLib.readIni(requestNovelsLib.getDownLoadIniFilePath(), f"{getRootUrl()}", "pageinfocount", _pageinfocount))
	except:
		requestNovelsLib.writeLogFile("无法获取匹配的数量", httpUrl=getRootUrl())
		traceback.print_exc()
	rootUrl = f"https://{getRootUrl()}/"
	
	requestUrl: requestNovelsLib.ResponseNovel = None
	try:
		requestUrl = requestNovelsLib.ResponseNovel(rootUrl)
	except:
		requestNovelsLib.writeLogFile("无法实现第一次请求", rootUrl)
		sys.stderr.write("无法实现首页请求")
		return None
	newFolder = requestNovelsLib.getUrlTargetFolder(getRootUrl())
	_appRequestNovelPage = None
	try:
		## 获取类型类型的请求页面
		runEndTime = requestNovelsLib.ayscRequestGetUrl(requestUrl, getNovelHome, getPageListNovel, getPageNextUrl, rootUrl, newFolder, __runMain, proLock=lock, nextRequestTimeSece=nextRequestTimeSece, workerThreadNumber=workerThreadNumber, titleSleepTime=titleSleepTime, getAppendRequestNovelPageFunctionPtr=None, endCall=endCall, novelMaxCount=__maxNovelCount)
		setRunTime(runEndTime)
	except:
		traceback.print_exc()
	return


if __name__ == '__main__':
	__runMain = True
	requestNovelsLib.initLib()
	modeRequestGetUrl()
